DOMAIN: Automotive Surveillance.
CONTEXT:Computer vision can be used to automate supervision and generate action appropriate action trigger if the event is predicted from the image of interest. For example a car moving on the road can be easily identified by a camera as make of the car, type, colour, number plates etc.
DATA DESCRIPTION:The Cars dataset contains 16,185 images of 196 classes of cars. The data is split into 8,144 training images and 8,041 testing images, where each class has been split roughly in a 50-50 split. Classes are typically at the level of Make, Model, Year, e.g. 2012 Tesla Model S or 2012 BMW M3 coupe.
Loading Libraries
from sklearn.preprocessing import StandardScaler # StandardScaler is used to resize the distribution of values,
# so that the mean of the observed values is 0 and the standard deviation is 1
import matplotlib.pyplot as plt # Matplotlib is a cross-platform, data visualization and graphical plotting library
import numpy as np # NumPy is a Python library used for working with arrays & linear algebra
import os # The OS module in Python provides functions for creating and removing a directory (folder),
# fetching its contents, changing and identifying the current directory, etc.
import pandas as pd # data processing, in this project used for CSV file I/O (e.g. pd.read_csv)
Mounting Google Drive, Data has been downloaded and stored in Google Drive. Using which user can access the supporting files like Annotations, Car Images and Car Names
from google.colab import drive
drive.mount('/content/drive')
Listing the Files and Folders available under Capstone Project
print(os.listdir('/content/drive/MyDrive/CapstoneProject/'))
# listdir() method in python is used to get the list of all files and directories in the specified directory
User Defined Function to create Correlation Plot Matrix
# User-defined function for "Correlation matrix"
def plotCorrelationMatrix(df, graphWidth): # 'df'-dataframe; 'graphwidth'-width (units = inches) are the inputs to this function
"""User Defined function to create Correlation matrix"""
filename = df.dataframeName # the dataframe 'df' name is assigned to 'filename' variable
df = df.dropna('columns') # drop columns with 'NaN'
df = df[[col for col in df if df[col].nunique() > 1]] # keep columns where there are more than 1 unique values
if df.shape[1] < 2: # checking if the dataframe has only one column, in such case no correlation can takes place
print(f'No correlation plots shown: The number of non-NaN or constant columns ({df.shape[1]}) is less than 2')
return
corr = df.corr() # corr() is used to find the pairwise correlation of all columns in the dataframe
plt.figure(num=None, figsize=(graphWidth, graphWidth), dpi=80, facecolor='w', edgecolor='k')
# Create a new figure, or activate an existing figure
# num = A unique identifier for the figure. 'None' = a new figure is created
# figsize = Width, height in inches. Can be (float, float)
# dpi = The resolution of the figure in dots-per-inch.
# facecolor = The background color.
# edgecolor = The border color.
corrMat = plt.matshow(corr, fignum = 1)
# matshow = Display an array as a matrix in a new figure window.
# The origin is set at the upper left hand corner and rows (first dimension of the array) are displayed horizontally.
# corr = The matrix to be displayed.
# fignum = If None, create a new figure window with automatic numbering.
# fignum = If a nonzero integer, draw into the figure with the given number (create it if it does not exist).
# fignum = If 0, use the current axes (or create one if it does not exist).
plt.xticks(range(len(corr.columns)), corr.columns, rotation=90)
# xticks = Get or set the current tick locations and labels of the x-axis
# range(len()) = to access each item by index in columns of corr
# corr.columns = The list of xlabel Text objects
# rotation = rotating xlabels
plt.yticks(range(len(corr.columns)), corr.columns)
# similar to above 'xticks', but with 'y'
plt.gca().xaxis.tick_bottom()
#
plt.colorbar(corrMat)
plt.title(f'Correlation Matrix for {filename}', fontsize=15)
plt.show()
print(plotCorrelationMatrix.__doc__)
# Scatter and density plots
def plotScatterMatrix(df, plotSize, textSize):
"""User Defined function to create Scatterpolt Matrix"""
df = df.select_dtypes(include =[np.number]) # keep only numerical columns
# Remove rows and columns that would lead to df being singular
df = df.dropna('columns')
df = df[[col for col in df if df[col].nunique() > 1]] # keep columns where there are more than 1 unique values
columnNames = list(df)
if len(columnNames) > 10: # reduce the number of columns for matrix inversion of kernel density plots
columnNames = columnNames[:10]
df = df[columnNames]
ax = pd.plotting.scatter_matrix(df, alpha=0.75, figsize=[plotSize, plotSize], diagonal='kde')
corrs = df.corr().values
for i, j in zip(*plt.np.triu_indices_from(ax, k = 1)):
ax[i, j].annotate('Corr. coef = %.3f' % corrs[i, j], (0.8, 0.2), xycoords='axes fraction', ha='center', va='center', size=textSize)
plt.suptitle('Scatter and Density Plot')
plt.show()
print(plotScatterMatrix.__doc__)
Exploring Test Annotation file
Loading TestAnnotation.csv to explore it
nRowsRead = None
# anno_test.csv has 8041 rows in reality, but we are only loading/previewing the first 1000 rows
test_anno_df = pd.read_csv('/content/drive/MyDrive/CapstoneProject/Annotations/TestAnnotation.csv')
test_anno_df.rename(columns={'Bounding Box coordinates': 'x0', 'Unnamed: 2': 'y0','Unnamed: 3': 'x1','Unnamed: 4': 'y1'}, inplace=True)
test_anno_df.dataframeName = 'TestAnnotation.csv'
nRow, nCol = test_anno_df.shape
print(f'There are {nRow} rows and {nCol} columns')
test_anno_df.head()
Checking Test Annotation files datatypes, there is no issue with the datatypes and every column is mapped properly
test_anno_df.info()
Counting Unique Classes in Test Annotations file
# counting unique values
n = len(pd.unique(test_anno_df['Image class']))
print(n)
plotCorrelationMatrix(test_anno_df, 8)
plotScatterMatrix(test_anno_df, 15, 10)
Exploring Train Annotation file
nRowsRead = None
# anno_test.csv has 8041 rows in reality, but we are only loading/previewing the first 1000 rows
train_anno_df = pd.read_csv('/content/drive/MyDrive/CapstoneProject/Annotations/TrainAnnotations.csv')
train_anno_df.rename(columns={'Bounding Box coordinates': 'x0', 'Unnamed: 2': 'y0','Unnamed: 3': 'x1','Unnamed: 4': 'y1'}, inplace=True)
train_anno_df.dataframeName = 'TrainAnnotation.csv'
nRow, nCol = train_anno_df.shape
print(f'There are {nRow} rows and {nCol} columns')
train_anno_df.head()
Checking Train Annotation files datatypes, there is no issue with the datatypes and every column is mapped properly
train_anno_df.info()
Counting Unique Classes in Train Annotations file
# counting unique values
n = len(pd.unique(train_anno_df['Image class']))
print(n)
plotCorrelationMatrix(train_anno_df, 8)
plotScatterMatrix(train_anno_df, 15, 10)
Exploring CarNames file
Loading TrainAnnotation.csv to explore it
nRowsRead = None
#Loading name of cars
carNameDF= pd.read_csv("/content/drive/MyDrive/CapstoneProject/CarNames.csv",header=None)
carNameDF.dataframeName = 'CarNames.csv'
nRow, nCol = carNameDF.shape
print(f'There are {nRow} rows and {nCol} columns')
carNameDF.rename(columns={0: 'CarNames'}, inplace=True)
carNameDF.info()
carNameDF.head()
Getting unqiue count
n = len(pd.unique(carNameDF['CarNames']))
print(n)
User defined function to create image dataframe
import os
import tensorflow
from keras.preprocessing import image
#from tqdm import tqdm
def createImageDF(imageSourceFolder,target_size_tuple,colormode,totalImagePerClassCap):
dfRows=[]
#l = glob.glob('/content/drive/MyDrive/CapstoneProject/CarImages/TrainImages/*/*')
for dirname,_, filenames in os.walk(imageSourceFolder):
limit=0
if len(filenames) > totalImagePerClassCap or totalImagePerClassCap == 0:
limit=len(filenames)
else:
limit=totalImagePerClassCap
for i in range(0,limit):
processedRows=[]
filename=filenames[i]
processedRows.append(filename)
pathSplits=dirname.split('/')
print(os.path.join(dirname, filename))
imgPath=os.path.join(dirname, filename)
print(imgPath)
img = image.load_img(imgPath)
if target_size_tuple!='':
img_scaled = image.load_img(imgPath, target_size=target_size_tuple, color_mode = colormode)
image_width_scale_fact=target_size_tuple[0]/img.width
image_height_scale_fact=target_size_tuple[1]/img.height
imgArray = image.img_to_array(img_scaled)
else:
img
image_width_scale_fact=img.width
image_height_scale_fact=img.height
#img = Image.open(imgPath)
imgArray = image.img_to_array(img)
#imgArray=np.asarray(img)
processedRows.append(imgArray)
processedRows.append(pathSplits[len(pathSplits)-1])
processedRows.append(image_width_scale_fact)
processedRows.append(image_height_scale_fact)
dfRows.append(processedRows)
return pd.DataFrame(dfRows, columns=["Image Name","image_array","model","image_width_scale_fact","image_height_scale_fact"])
User Defined function to load the single score image
def loadSingleScoringImage(imageSourcePath,target_size_tuple,colormode):
dfRows=[]
processedRow=[]
imgPath=imageSourcePath
print(imgPath)
img = image.load_img(imgPath)
pathSplits=imgPath.split('/')
if target_size_tuple!='':
img_scaled = image.load_img(imgPath, target_size=target_size_tuple, color_mode = colormode)
image_width_scale_fact=target_size_tuple[0]/img.width
image_height_scale_fact=target_size_tuple[1]/img.height
imgArray = image.img_to_array(img_scaled)
else:
img
image_width_scale_fact=img.width
image_height_scale_fact=img.height
#img = Image.open(imgPath)
imgArray = image.img_to_array(img)
#imgArray=np.asarray(img)
processedRow.append(pathSplits[len(pathSplits)-1])
processedRow.append(imgArray)
processedRow.append(pathSplits[len(pathSplits)-2])
processedRow.append(image_width_scale_fact)
processedRow.append(image_height_scale_fact)
dfRows.append(processedRow)
return pd.DataFrame(dfRows, columns=["Image Name","image_array","model","image_width_scale_fact","image_height_scale_fact"])
# import glob
# TrainImage = glob.glob('/content/drive/MyDrive/CapstoneProject/CarImages/TrainImages/*')
# FirstTime = 0
# for ImgFolder in TrainImage:
# try:
# #print(ImgFolder)
# trainImageDF=createImageDF(ImgFolder,'',"rgb",5)
# if FirstTime == 0:
# mergedTrainImageDF = trainImageDF
# FirstTime = FirstTime + 1
# else:
# mergedTrainImageDF = mergedTrainImageDF.append(trainImageDF)
# except:
# print("Success")
#As GPU system got full, I have loaded the sample classes and created Image dataframe
trainImageDF=createImageDF('/content/drive/MyDrive/CapstoneProject/CarImages/TrainImages/Audi 100 Sedan 1994','',"rgb",2)
trainImageDF.shape
Merging TrainImageDF and Train Annotations
trainImageDF.head()
train_anno_df.head()
Merging the trainImageDF and train_anno_df using the common filed - Image Name
# Lodaing annotation details for images from train_anno_df i.e. train annotations
trainMergedImageDF=pd.merge(trainImageDF,train_anno_df, on=['Image Name'])
trainMergedImageDF.head()
Function to create bounding box on image
import matplotlib.pyplot as plt
import matplotlib.patches as patches
def drawBoxedImages(dataFrame,featureCol,x0,y0,x1,y1,boxlinewidth,boxedgecolor):
fig,ax=plt.subplots(len(dataFrame))
fig.set_size_inches(30,30)
# Display the image
for index, row in dataFrame.iterrows():
data=row[featureCol]
img = image.array_to_img(data)
ax[index].imshow(img)
image_height, image_width, _ = data.shape
rect=[]
rect.append(patches.Rectangle((row[x0], row[y0]), row[x1] - row[x0], row[y1] - row[y0], linewidth=boxlinewidth, edgecolor=boxedgecolor, facecolor='none'))
# Add the patch to the Axes
for l in range(0,len(rect)):
ax[index].add_patch(rect[l])
plt.show()
def drawImages(dataFrame,featureCol):
fig,ax=plt.subplots(len(dataFrame))
fig.set_size_inches(30,30)
# Display the image
for index, row in dataFrame.iterrows():
data=row[featureCol]
img = image.array_to_img(data)
ax[index].imshow(img)
plt.show()
drawBoxedImages(trainMergedImageDF.head(5),'image_array','x0','y0','x1','y1',2,'r')
testImageDF=createImageDF('/content/drive/MyDrive/CapstoneProject/CarImages/TestImages/Audi 100 Sedan 1994','',"rgb",2)
testImageDF.shape
Merging the testImageDF and test_anno_df using the common filed - Image Name
testImageDF.head()
test_anno_df.head()
# Lodaing annotation details for images from train_anno_df i.e. train annotations
testMergedImageDF=pd.merge(testImageDF,test_anno_df, on=['Image Name'])
testMergedImageDF.head(10)
drawBoxedImages(testMergedImageDF.head(10),'image_array','x0','y0','x1','y1',2,'r')
import matplotlib.pyplot as plt
import numpy as np
import torch
import torch.nn as nn
import torch.optim as optim
import torchvision
import torchvision.models as models
import torchvision.transforms as transforms
import time
import os
import PIL.Image as Image
from IPython.display import display
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
print(device)
print(torch.cuda.get_device_name(device))
dataset_dir = "/content/drive/MyDrive/CapstoneProject/CarImages/"
train_tfms = transforms.Compose([transforms.Resize((400, 400)),
transforms.RandomHorizontalFlip(),
transforms.RandomRotation(15),
transforms.ToTensor(),
transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])
test_tfms = transforms.Compose([transforms.Resize((400, 400)),
transforms.ToTensor(),
transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])
dataset = torchvision.datasets.ImageFolder(root=dataset_dir+"train", transform = train_tfms)
trainloader = torch.utils.data.DataLoader(dataset, batch_size = 32, shuffle=True, num_workers = 2)
dataset2 = torchvision.datasets.ImageFolder(root=dataset_dir+"test", transform = test_tfms)
testloader = torch.utils.data.DataLoader(dataset2, batch_size = 32, shuffle=False, num_workers = 2)
Creating the user defined function for train_model
def train_model(model, criterion, optimizer, scheduler, n_epochs = 5):
losses = []
accuracies = []
test_accuracies = []
# set the model to train mode initially
model.train()
for epoch in range(n_epochs):
since = time.time()
running_loss = 0.0
running_correct = 0.0
for i, data in enumerate(trainloader, 0):
# get the inputs and assign them to cuda
inputs, labels = data
#inputs = inputs.to(device).half() # uncomment for half precision model
inputs = inputs.to(device)
labels = labels.to(device)
optimizer.zero_grad()
# forward + backward + optimize
outputs = model(inputs)
_, predicted = torch.max(outputs.data, 1)
loss = criterion(outputs, labels)
loss.backward()
optimizer.step()
# calculate the loss/acc later
running_loss += loss.item()
running_correct += (labels==predicted).sum().item()
epoch_duration = time.time()-since
epoch_loss = running_loss/len(trainloader)
epoch_acc = 100/32*running_correct/len(trainloader)
print("Epoch %s, duration: %d s, loss: %.4f, acc: %.4f" % (epoch+1, epoch_duration, epoch_loss, epoch_acc))
losses.append(epoch_loss)
accuracies.append(epoch_acc)
# switch the model to eval mode to evaluate on test data
model.eval()
test_acc = eval_model(model)
test_accuracies.append(test_acc)
# re-set the model to train mode after validating
model.train()
scheduler.step(test_acc)
since = time.time()
print('Finished Training')
return model, losses, accuracies, test_accuracies
def eval_model(model):
correct = 0.0
total = 0.0
with torch.no_grad():
for i, data in enumerate(testloader, 0):
images, labels = data
#images = images.to(device).half() # uncomment for half precision model
images = images.to(device)
labels = labels.to(device)
outputs = model_ft(images)
_, predicted = torch.max(outputs.data, 1)
total += labels.size(0)
correct += (predicted == labels).sum().item()
test_acc = 100.0 * correct / total
print('Accuracy of the network on the test images: %d %%' % (
test_acc))
return test_acc
model_ft = models.resnet34(pretrained=True)
num_ftrs = model_ft.fc.in_features
model_ft.fc = nn.Linear(num_ftrs, 196)
model_ft = model_ft.to(device)
criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(model_ft.parameters(), lr=0.01, momentum=0.9)
lrscheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='max', patience=3, threshold = 0.9)
model_ft, training_losses, training_accs, test_accs = train_model(model_ft, criterion, optimizer, lrscheduler, n_epochs=10)
# plot the stats
f, axarr = plt.subplots(2,2, figsize = (12, 8))
axarr[0, 0].plot(training_losses)
axarr[0, 0].set_title("Training loss")
axarr[0, 1].plot(training_accs)
axarr[0, 1].set_title("Training acc")
axarr[1, 0].plot(test_accs)
axarr[1, 0].set_title("Test acc")
# tie the class indices to their names
def find_classes(dir):
classes = os.listdir(dir)
classes.sort()
class_to_idx = {classes[i]: i for i in range(len(classes))}
return classes, class_to_idx
classes, c_to_idx = find_classes(dataset_dir+"train")
# test the model on random images
dataset_dir = "/content/drive/MyDrive/CapstoneProject/CarImages"
torch.save(model_ft,"/content/drive/MyDrive/CapstoneProject/CarImages/working/model.h5")
# switch the model to evaluation mode to make dropout and batch norm work in eval mode
model_ft.eval()
# transforms for the input image
loader = transforms.Compose([transforms.Resize((400, 400)),
transforms.ToTensor(),
transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])
image = Image.open(dataset_dir+"test/Mercedes-Benz C-Class Sedan 2012/01977.jpg")
image = loader(image).float()
image = torch.autograd.Variable(image, requires_grad=True)
image = image.unsqueeze(0)
image = image.cuda()
output = model_ft(image)
conf, predicted = torch.max(output.data, 1)
# get the class name of the prediction
display(Image.open(dataset_dir+"test/Mercedes-Benz C-Class Sedan 2012/01977.jpg"))
print(classes[predicted.item()], "confidence: ", conf.item())
dataset_dir = "/content/drive/MyDrive/CapstoneProject/CarImages/test"
import os
listFolders = os.listdir(dataset_dir)
import glob
classNames = []
filePaths=[]
for folder in listFolders:
i=0
for file in glob.glob(dataset_dir+"/"+folder+"/*.jpg"):
if(i==5):
break
classNames.append(folder)
filePaths.append(file)
i=i+1
count = 0
for imgPath in filePaths:
# transforms for the input image
try:
loader = transforms.Compose([transforms.Resize((400, 400)),
transforms.ToTensor(),
transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])
image = Image.open(imgPath)
if(image is None):
continue
image = loader(image).float()
image = torch.autograd.Variable(image, requires_grad=True)
image = image.unsqueeze(0)
image = image.cuda()
output = model_ft(image)
conf, predicted = torch.max(output.data, 1)
display(Image.open(imgPath))
print(classes[predicted.item()], "confidence: ", conf.item())
count += 1
if count >= 10:
break
except:
print("Error")